#!/bin/sh

cat README.md | grep "^\*" | cut -f 2 -d " " | egrep -v "imgur\.com|github\.com|\.yara|youtu\.be" | sed "s/twitter\.com/nitter.net/g" | egrep "pdf" | while read articleurl
do
	articlefilename="$(printf "%s" "${articleurl}" | tr ":/" "__")"
	if [ ! -f "articles/${articlefilename}" ]
	then
		wget --user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" -O "articles/${articlefilename}" "${articleurl}"
	else
		printf "%s has already been grabbed\n" "articles/${articlefilename}"
	fi
done
cat README.md | grep "^\*" | cut -f 2 -d " " | egrep -v "imgur\.com|github\.com|\.yara|youtu\.be" | sed "s/twitter\.com/nitter.net/g" | egrep "jpg|jpeg" | while read articleurl
do
	articlefilename="$(printf "%s" "${articleurl}" | tr ":/" "__")"
	if [ ! -f "articles/${articlefilename}" ]
	then
		wget --user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" -O "articles/${articlefilename}" "${articleurl}"
	else
		printf "%s has already been grabbed\n" "articles/${articlefilename}"
	fi
done
cat README.md | grep "^\*" | cut -f 2 -d " " | egrep -v "imgur\.com|github\.com|\.yara|youtu\.be" | sed "s/twitter\.com/nitter.net/g" | egrep "htm" | while read articleurl
do
	articlefilename="$(printf "%s" "${articleurl}" | tr ":/" "__" | cut -f 1 -d "#")"
	if [ ! -f "articles/${articlefilename}" ]
	then
		wget --user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" -O "articles/${articlefilename}" "${articleurl}"
	else
		printf "%s has already been grabbed\n" "articles/${articlefilename}"
	fi
done
cat README.md | grep "^\*" | cut -f 2 -d " " | egrep -v "imgur\.com|github\.com|\.yara|youtu\.be" | sed "s/twitter\.com/nitter.net/g" | egrep -v "pdf|jpg|jpeg|htm" | while read articleurl
do
	articlefilename="$(printf "%s" "${articleurl}" | tr ":/" "__" | cut -f 1 -d "#")"
	if [ ! -f "articles/${articlefilename}.html" ]
	then
		wget --user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" -O "articles/${articlefilename}.html" "${articleurl}"
	else
		printf "%s has already been grabbed\n" "articles/${articlefilename}.html"
	fi
done
cat README.md | grep "^\*" | cut -f 2 -d " " | egrep "github\.com" | egrep "raw" | egrep "pdf|md" | while read articleurl
do
	articlefilename="$(printf "%s" "${articleurl}" | tr ":/" "__" | cut -f 1 -d "#")"
	if [ ! -f "articles/${articlefilename}" ]
	then
		wget --user-agent="Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/74.0.3729.169 Safari/537.36" -O "articles/${articlefilename}" "${articleurl}"
	else
		printf "%s has already been grabbed\n" "articles/${articlefilename}"
	fi
done
